{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from time import time\n",
    "col_names = [\"duration\",\"protocol_type\",\"service\",\"flag\",\"src_bytes\",\n",
    "    \"dst_bytes\",\"land\",\"wrong_fragment\",\"urgent\",\"hot\",\"num_failed_logins\",\n",
    "    \"logged_in\",\"num_compromised\",\"root_shell\",\"su_attempted\",\"num_root\",\n",
    "    \"num_file_creations\",\"num_shells\",\"num_access_files\",\"num_outbound_cmds\",\n",
    "    \"is_host_login\",\"is_guest_login\",\"count\",\"srv_count\",\"serror_rate\",\n",
    "    \"srv_serror_rate\",\"rerror_rate\",\"srv_rerror_rate\",\"same_srv_rate\",\n",
    "    \"diff_srv_rate\",\"srv_diff_host_rate\",\"dst_host_count\",\"dst_host_srv_count\",\n",
    "    \"dst_host_same_srv_rate\",\"dst_host_diff_srv_rate\",\"dst_host_same_src_port_rate\",\n",
    "    \"dst_host_srv_diff_host_rate\",\"dst_host_serror_rate\",\"dst_host_srv_serror_rate\",\n",
    "    \"dst_host_rerror_rate\",\"dst_host_srv_rerror_rate\",\"label\"]\n",
    "DoS = ['back.','land.','neptune.','pod.','smurf.', 'teardrop.']\n",
    "R2L = ['ftp_write.','guess_passwd.','imap.','multihop.','phf.','spy.','warezclient.','warezmaster.']\n",
    "U2R = ['buffer-overflow.','loadmodule.','perl.','rootkit.']\n",
    "Probe = ['ipsweep.','nmap.','portsweep.','satan.']\n",
    "kdd_data = pd.read_csv(\"kddcup.data.corrected\", header=None, names = col_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>protocol_type</th>\n",
       "      <th>service</th>\n",
       "      <th>flag</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>...</th>\n",
       "      <th>dst_host_srv_count</th>\n",
       "      <th>dst_host_same_srv_rate</th>\n",
       "      <th>dst_host_diff_srv_rate</th>\n",
       "      <th>dst_host_same_src_port_rate</th>\n",
       "      <th>dst_host_srv_diff_host_rate</th>\n",
       "      <th>dst_host_serror_rate</th>\n",
       "      <th>dst_host_srv_serror_rate</th>\n",
       "      <th>dst_host_rerror_rate</th>\n",
       "      <th>dst_host_srv_rerror_rate</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>215</td>\n",
       "      <td>45076</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>normal.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>162</td>\n",
       "      <td>4528</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>normal.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>236</td>\n",
       "      <td>1228</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>normal.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>233</td>\n",
       "      <td>2032</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>normal.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>239</td>\n",
       "      <td>486</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>normal.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   duration protocol_type service flag  src_bytes  dst_bytes  land  \\\n",
       "0         0           tcp    http   SF        215      45076     0   \n",
       "1         0           tcp    http   SF        162       4528     0   \n",
       "2         0           tcp    http   SF        236       1228     0   \n",
       "3         0           tcp    http   SF        233       2032     0   \n",
       "4         0           tcp    http   SF        239        486     0   \n",
       "\n",
       "   wrong_fragment  urgent  hot   ...     dst_host_srv_count  \\\n",
       "0               0       0    0   ...                      0   \n",
       "1               0       0    0   ...                      1   \n",
       "2               0       0    0   ...                      2   \n",
       "3               0       0    0   ...                      3   \n",
       "4               0       0    0   ...                      4   \n",
       "\n",
       "   dst_host_same_srv_rate  dst_host_diff_srv_rate  \\\n",
       "0                     0.0                     0.0   \n",
       "1                     1.0                     0.0   \n",
       "2                     1.0                     0.0   \n",
       "3                     1.0                     0.0   \n",
       "4                     1.0                     0.0   \n",
       "\n",
       "   dst_host_same_src_port_rate  dst_host_srv_diff_host_rate  \\\n",
       "0                         0.00                          0.0   \n",
       "1                         1.00                          0.0   \n",
       "2                         0.50                          0.0   \n",
       "3                         0.33                          0.0   \n",
       "4                         0.25                          0.0   \n",
       "\n",
       "   dst_host_serror_rate  dst_host_srv_serror_rate  dst_host_rerror_rate  \\\n",
       "0                   0.0                       0.0                   0.0   \n",
       "1                   0.0                       0.0                   0.0   \n",
       "2                   0.0                       0.0                   0.0   \n",
       "3                   0.0                       0.0                   0.0   \n",
       "4                   0.0                       0.0                   0.0   \n",
       "\n",
       "   dst_host_srv_rerror_rate    label  \n",
       "0                       0.0  normal.  \n",
       "1                       0.0  normal.  \n",
       "2                       0.0  normal.  \n",
       "3                       0.0  normal.  \n",
       "4                       0.0  normal.  \n",
       "\n",
       "[5 rows x 42 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kdd_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4898431"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(kdd_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "icmp    2833545\n",
      "tcp     1870598\n",
      "udp      194288\n",
      "Name: protocol_type, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(kdd_data['protocol_type'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ecr_i          2811660\n",
      "private        1100831\n",
      "http            623091\n",
      "smtp             96554\n",
      "other            72653\n",
      "domain_u         57782\n",
      "ftp_data         40697\n",
      "eco_i            16338\n",
      "finger            6891\n",
      "urp_i             5378\n",
      "ftp               5214\n",
      "telnet            4277\n",
      "ntp_u             3833\n",
      "auth              3382\n",
      "pop_3             1981\n",
      "time              1579\n",
      "domain            1113\n",
      "Z39_50            1078\n",
      "gopher            1077\n",
      "mtp               1076\n",
      "ssh               1075\n",
      "whois             1073\n",
      "remote_job        1073\n",
      "rje               1070\n",
      "imap4             1069\n",
      "link              1069\n",
      "ctf               1068\n",
      "name              1067\n",
      "supdup            1060\n",
      "discard           1059\n",
      "                ...   \n",
      "vmnet             1053\n",
      "netbios_dgm       1052\n",
      "sql_net           1052\n",
      "iso_tsap          1052\n",
      "shell             1051\n",
      "csnet_ns          1051\n",
      "klogin            1050\n",
      "hostnames         1050\n",
      "bgp               1047\n",
      "exec              1045\n",
      "login             1045\n",
      "printer           1045\n",
      "http_443          1044\n",
      "efs               1042\n",
      "uucp              1041\n",
      "ldap              1041\n",
      "kshell            1040\n",
      "nnsp              1038\n",
      "courier           1021\n",
      "IRC                521\n",
      "urh_i              148\n",
      "X11                135\n",
      "tim_i               12\n",
      "red_i                9\n",
      "pm_dump              5\n",
      "tftp_u               3\n",
      "harvest              2\n",
      "aol                  2\n",
      "http_8001            2\n",
      "http_2784            1\n",
      "Name: service, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(kdd_data['service'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "smurf.              2807886\n",
      "neptune.            1072017\n",
      "normal.              972781\n",
      "satan.                15892\n",
      "ipsweep.              12481\n",
      "portsweep.            10413\n",
      "nmap.                  2316\n",
      "back.                  2203\n",
      "warezclient.           1020\n",
      "teardrop.               979\n",
      "pod.                    264\n",
      "guess_passwd.            53\n",
      "buffer_overflow.         30\n",
      "land.                    21\n",
      "warezmaster.             20\n",
      "imap.                    12\n",
      "rootkit.                 10\n",
      "loadmodule.               9\n",
      "ftp_write.                8\n",
      "multihop.                 7\n",
      "phf.                      4\n",
      "perl.                     3\n",
      "spy.                      2\n",
      "Name: label, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(kdd_data['label'].value_counts())\n",
    "#print(len(kdd_data['service'].value_counts()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SF        3744328\n",
      "S0         869829\n",
      "REJ        268874\n",
      "RSTR         8094\n",
      "RSTO         5344\n",
      "SH           1040\n",
      "S1            532\n",
      "S2            161\n",
      "RSTOS0        122\n",
      "OTH            57\n",
      "S3             50\n",
      "Name: flag, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(kdd_data['flag'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#OneHotEncode protcol_type and flag. \n",
    "cols_to_transform = ['protocol_type', 'flag']\n",
    "df_with_dummies = pd.get_dummies(columns = cols_to_transform, data = kdd_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>service</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>num_failed_logins</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>...</th>\n",
       "      <th>flag_REJ</th>\n",
       "      <th>flag_RSTO</th>\n",
       "      <th>flag_RSTOS0</th>\n",
       "      <th>flag_RSTR</th>\n",
       "      <th>flag_S0</th>\n",
       "      <th>flag_S1</th>\n",
       "      <th>flag_S2</th>\n",
       "      <th>flag_S3</th>\n",
       "      <th>flag_SF</th>\n",
       "      <th>flag_SH</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>http</td>\n",
       "      <td>215</td>\n",
       "      <td>45076</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>http</td>\n",
       "      <td>162</td>\n",
       "      <td>4528</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>http</td>\n",
       "      <td>236</td>\n",
       "      <td>1228</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>http</td>\n",
       "      <td>233</td>\n",
       "      <td>2032</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>http</td>\n",
       "      <td>239</td>\n",
       "      <td>486</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 54 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   duration service  src_bytes  dst_bytes  land  wrong_fragment  urgent  hot  \\\n",
       "0         0    http        215      45076     0               0       0    0   \n",
       "1         0    http        162       4528     0               0       0    0   \n",
       "2         0    http        236       1228     0               0       0    0   \n",
       "3         0    http        233       2032     0               0       0    0   \n",
       "4         0    http        239        486     0               0       0    0   \n",
       "\n",
       "   num_failed_logins  logged_in   ...     flag_REJ  flag_RSTO  flag_RSTOS0  \\\n",
       "0                  0          1   ...            0          0            0   \n",
       "1                  0          1   ...            0          0            0   \n",
       "2                  0          1   ...            0          0            0   \n",
       "3                  0          1   ...            0          0            0   \n",
       "4                  0          1   ...            0          0            0   \n",
       "\n",
       "   flag_RSTR  flag_S0  flag_S1  flag_S2  flag_S3  flag_SF  flag_SH  \n",
       "0          0        0        0        0        0        1        0  \n",
       "1          0        0        0        0        0        1        0  \n",
       "2          0        0        0        0        0        1        0  \n",
       "3          0        0        0        0        0        1        0  \n",
       "4          0        0        0        0        0        1        0  \n",
       "\n",
       "[5 rows x 54 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_with_dummies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label value count before grouping = smurf.              2807886\n",
      "neptune.            1072017\n",
      "normal.              972781\n",
      "satan.                15892\n",
      "ipsweep.              12481\n",
      "portsweep.            10413\n",
      "nmap.                  2316\n",
      "back.                  2203\n",
      "warezclient.           1020\n",
      "teardrop.               979\n",
      "pod.                    264\n",
      "guess_passwd.            53\n",
      "buffer_overflow.         30\n",
      "land.                    21\n",
      "warezmaster.             20\n",
      "imap.                    12\n",
      "rootkit.                 10\n",
      "loadmodule.               9\n",
      "ftp_write.                8\n",
      "multihop.                 7\n",
      "phf.                      4\n",
      "perl.                     3\n",
      "spy.                      2\n",
      "Name: label, dtype: int64\n",
      "Label value count after grouping = DoS       3883370\n",
      "Normal     972811\n",
      "Probe       41102\n",
      "R2L          1126\n",
      "U2R            22\n",
      "Name: label, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "#Group label into 4 attack types and OneHotEncode\n",
    "print(\"Label value count before grouping = {}\".format(df_with_dummies['label'].value_counts()))\n",
    "\n",
    "def classifyAttack(x):\n",
    "    if x in DoS:\n",
    "        return 'DoS'\n",
    "    elif x in R2L:\n",
    "        return 'R2L'\n",
    "    elif x in U2R:\n",
    "        return 'U2R'\n",
    "    elif x in Probe:\n",
    "        return 'Probe'\n",
    "    else:\n",
    "        return 'Normal'\n",
    "\n",
    "for i in range(len(df_with_dummies)):\n",
    "    df_with_dummies.set_value(i,'label',classifyAttack(df_with_dummies['label'][i]))\n",
    "    \n",
    "print(\"Label value count after grouping = {}\".format(df_with_dummies['label'].value_counts()))\n",
    "\n",
    "cols_to_transform = ['label']\n",
    "df_with_dummies = pd.get_dummies(columns = cols_to_transform, data = df_with_dummies)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#LabelEncode all 70 types of service attribute\n",
    "from sklearn import preprocessing \n",
    "le = preprocessing.LabelEncoder()\n",
    "#from collections import defaultdict\n",
    "#d = defaultdict(preprocessing.LabelEncoder)\n",
    "\n",
    "df_with_dummies = df_with_dummies.apply(le.fit_transform)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>service</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>num_failed_logins</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>...</th>\n",
       "      <th>flag_S1</th>\n",
       "      <th>flag_S2</th>\n",
       "      <th>flag_S3</th>\n",
       "      <th>flag_SF</th>\n",
       "      <th>flag_SH</th>\n",
       "      <th>label_DoS</th>\n",
       "      <th>label_Normal</th>\n",
       "      <th>label_Probe</th>\n",
       "      <th>label_R2L</th>\n",
       "      <th>label_U2R</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>213</td>\n",
       "      <td>19600</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>160</td>\n",
       "      <td>4511</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>234</td>\n",
       "      <td>1221</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>231</td>\n",
       "      <td>2025</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>237</td>\n",
       "      <td>479</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 58 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   duration  service  src_bytes  dst_bytes  land  wrong_fragment  urgent  hot  \\\n",
       "0         0       24        213      19600     0               0       0    0   \n",
       "1         0       24        160       4511     0               0       0    0   \n",
       "2         0       24        234       1221     0               0       0    0   \n",
       "3         0       24        231       2025     0               0       0    0   \n",
       "4         0       24        237        479     0               0       0    0   \n",
       "\n",
       "   num_failed_logins  logged_in    ...      flag_S1  flag_S2  flag_S3  \\\n",
       "0                  0          1    ...            0        0        0   \n",
       "1                  0          1    ...            0        0        0   \n",
       "2                  0          1    ...            0        0        0   \n",
       "3                  0          1    ...            0        0        0   \n",
       "4                  0          1    ...            0        0        0   \n",
       "\n",
       "   flag_SF  flag_SH  label_DoS  label_Normal  label_Probe  label_R2L  \\\n",
       "0        1        0          0             1            0          0   \n",
       "1        1        0          0             1            0          0   \n",
       "2        1        0          0             1            0          0   \n",
       "3        1        0          0             1            0          0   \n",
       "4        1        0          0             1            0          0   \n",
       "\n",
       "   label_U2R  \n",
       "0          0  \n",
       "1          0  \n",
       "2          0  \n",
       "3          0  \n",
       "4          0  \n",
       "\n",
       "[5 rows x 58 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_with_dummies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['duration',\n",
       " 'service',\n",
       " 'src_bytes',\n",
       " 'dst_bytes',\n",
       " 'land',\n",
       " 'wrong_fragment',\n",
       " 'urgent',\n",
       " 'hot',\n",
       " 'num_failed_logins',\n",
       " 'logged_in',\n",
       " 'num_compromised',\n",
       " 'root_shell',\n",
       " 'su_attempted',\n",
       " 'num_root',\n",
       " 'num_file_creations',\n",
       " 'num_shells',\n",
       " 'num_access_files',\n",
       " 'num_outbound_cmds',\n",
       " 'is_host_login',\n",
       " 'is_guest_login',\n",
       " 'count',\n",
       " 'srv_count',\n",
       " 'serror_rate',\n",
       " 'srv_serror_rate',\n",
       " 'rerror_rate',\n",
       " 'srv_rerror_rate',\n",
       " 'same_srv_rate',\n",
       " 'diff_srv_rate',\n",
       " 'srv_diff_host_rate',\n",
       " 'dst_host_count',\n",
       " 'dst_host_srv_count',\n",
       " 'dst_host_same_srv_rate',\n",
       " 'dst_host_diff_srv_rate',\n",
       " 'dst_host_same_src_port_rate',\n",
       " 'dst_host_srv_diff_host_rate',\n",
       " 'dst_host_serror_rate',\n",
       " 'dst_host_srv_serror_rate',\n",
       " 'dst_host_rerror_rate',\n",
       " 'dst_host_srv_rerror_rate',\n",
       " 'protocol_type_icmp',\n",
       " 'protocol_type_tcp',\n",
       " 'protocol_type_udp',\n",
       " 'flag_OTH',\n",
       " 'flag_REJ',\n",
       " 'flag_RSTO',\n",
       " 'flag_RSTOS0',\n",
       " 'flag_RSTR',\n",
       " 'flag_S0',\n",
       " 'flag_S1',\n",
       " 'flag_S2',\n",
       " 'flag_S3',\n",
       " 'flag_SF',\n",
       " 'flag_SH',\n",
       " 'label_DoS',\n",
       " 'label_Normal',\n",
       " 'label_Probe',\n",
       " 'label_R2L',\n",
       " 'label_U2R']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(df_with_dummies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>service</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>num_failed_logins</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>...</th>\n",
       "      <th>flag_S1</th>\n",
       "      <th>flag_S2</th>\n",
       "      <th>flag_S3</th>\n",
       "      <th>flag_SF</th>\n",
       "      <th>flag_SH</th>\n",
       "      <th>label_DoS</th>\n",
       "      <th>label_Normal</th>\n",
       "      <th>label_Probe</th>\n",
       "      <th>label_R2L</th>\n",
       "      <th>label_U2R</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.347826</td>\n",
       "      <td>0.029608</td>\n",
       "      <td>0.911967</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.347826</td>\n",
       "      <td>0.022241</td>\n",
       "      <td>0.209892</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.347826</td>\n",
       "      <td>0.032527</td>\n",
       "      <td>0.056812</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.347826</td>\n",
       "      <td>0.032110</td>\n",
       "      <td>0.094221</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.347826</td>\n",
       "      <td>0.032944</td>\n",
       "      <td>0.022287</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 58 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   duration   service  src_bytes  dst_bytes  land  wrong_fragment  urgent  \\\n",
       "0       0.0  0.347826   0.029608   0.911967   0.0             0.0     0.0   \n",
       "1       0.0  0.347826   0.022241   0.209892   0.0             0.0     0.0   \n",
       "2       0.0  0.347826   0.032527   0.056812   0.0             0.0     0.0   \n",
       "3       0.0  0.347826   0.032110   0.094221   0.0             0.0     0.0   \n",
       "4       0.0  0.347826   0.032944   0.022287   0.0             0.0     0.0   \n",
       "\n",
       "   hot  num_failed_logins  logged_in    ...      flag_S1  flag_S2  flag_S3  \\\n",
       "0  0.0                0.0        1.0    ...          0.0      0.0      0.0   \n",
       "1  0.0                0.0        1.0    ...          0.0      0.0      0.0   \n",
       "2  0.0                0.0        1.0    ...          0.0      0.0      0.0   \n",
       "3  0.0                0.0        1.0    ...          0.0      0.0      0.0   \n",
       "4  0.0                0.0        1.0    ...          0.0      0.0      0.0   \n",
       "\n",
       "   flag_SF  flag_SH  label_DoS  label_Normal  label_Probe  label_R2L  \\\n",
       "0      1.0      0.0        0.0           1.0          0.0        0.0   \n",
       "1      1.0      0.0        0.0           1.0          0.0        0.0   \n",
       "2      1.0      0.0        0.0           1.0          0.0        0.0   \n",
       "3      1.0      0.0        0.0           1.0          0.0        0.0   \n",
       "4      1.0      0.0        0.0           1.0          0.0        0.0   \n",
       "\n",
       "   label_U2R  \n",
       "0        0.0  \n",
       "1        0.0  \n",
       "2        0.0  \n",
       "3        0.0  \n",
       "4        0.0  \n",
       "\n",
       "[5 rows x 58 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_column_list = list(df_with_dummies)\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "min_max_scaler = preprocessing.MinMaxScaler(feature_range=(0,1))\n",
    "np_scaled = min_max_scaler.fit_transform(df_with_dummies)\n",
    "df_normalized = pd.DataFrame(np_scaled,columns=new_column_list)\n",
    "df_normalized.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>service</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>num_failed_logins</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>...</th>\n",
       "      <th>flag_S1</th>\n",
       "      <th>flag_S2</th>\n",
       "      <th>flag_S3</th>\n",
       "      <th>flag_SF</th>\n",
       "      <th>flag_SH</th>\n",
       "      <th>label_DoS</th>\n",
       "      <th>label_Normal</th>\n",
       "      <th>label_Probe</th>\n",
       "      <th>label_R2L</th>\n",
       "      <th>label_U2R</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>4.898431e+06</td>\n",
       "      <td>4.898431e+06</td>\n",
       "      <td>4.898431e+06</td>\n",
       "      <td>4.898431e+06</td>\n",
       "      <td>4.898431e+06</td>\n",
       "      <td>4.898431e+06</td>\n",
       "      <td>4.898431e+06</td>\n",
       "      <td>4.898431e+06</td>\n",
       "      <td>4.898431e+06</td>\n",
       "      <td>4.898431e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>4.898431e+06</td>\n",
       "      <td>4.898431e+06</td>\n",
       "      <td>4.898431e+06</td>\n",
       "      <td>4.898431e+06</td>\n",
       "      <td>4.898431e+06</td>\n",
       "      <td>4.898431e+06</td>\n",
       "      <td>4.898431e+06</td>\n",
       "      <td>4.898431e+06</td>\n",
       "      <td>4.898431e+06</td>\n",
       "      <td>4.898431e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.406568e-03</td>\n",
       "      <td>3.669251e-01</td>\n",
       "      <td>8.647840e-02</td>\n",
       "      <td>1.995741e-02</td>\n",
       "      <td>5.716116e-06</td>\n",
       "      <td>2.253783e-04</td>\n",
       "      <td>1.184053e-06</td>\n",
       "      <td>4.127078e-04</td>\n",
       "      <td>6.410216e-06</td>\n",
       "      <td>1.435290e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>1.086062e-04</td>\n",
       "      <td>3.286767e-05</td>\n",
       "      <td>1.020735e-05</td>\n",
       "      <td>7.643933e-01</td>\n",
       "      <td>2.123129e-04</td>\n",
       "      <td>7.927783e-01</td>\n",
       "      <td>1.985964e-01</td>\n",
       "      <td>8.390850e-03</td>\n",
       "      <td>2.298695e-04</td>\n",
       "      <td>4.491234e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>4.117306e-02</td>\n",
       "      <td>2.149888e-01</td>\n",
       "      <td>7.927254e-02</td>\n",
       "      <td>8.926702e-02</td>\n",
       "      <td>2.390833e-03</td>\n",
       "      <td>1.454819e-02</td>\n",
       "      <td>7.825853e-04</td>\n",
       "      <td>1.519020e-02</td>\n",
       "      <td>1.459882e-03</td>\n",
       "      <td>3.506116e-01</td>\n",
       "      <td>...</td>\n",
       "      <td>1.042087e-02</td>\n",
       "      <td>5.732939e-03</td>\n",
       "      <td>3.194878e-03</td>\n",
       "      <td>4.243774e-01</td>\n",
       "      <td>1.456941e-02</td>\n",
       "      <td>4.053158e-01</td>\n",
       "      <td>3.989435e-01</td>\n",
       "      <td>9.121648e-02</td>\n",
       "      <td>1.515971e-02</td>\n",
       "      <td>2.119249e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.173913e-01</td>\n",
       "      <td>5.977203e-03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.173913e-01</td>\n",
       "      <td>7.200445e-02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>6.666667e-01</td>\n",
       "      <td>1.431749e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 58 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           duration       service     src_bytes     dst_bytes          land  \\\n",
       "count  4.898431e+06  4.898431e+06  4.898431e+06  4.898431e+06  4.898431e+06   \n",
       "mean   3.406568e-03  3.669251e-01  8.647840e-02  1.995741e-02  5.716116e-06   \n",
       "std    4.117306e-02  2.149888e-01  7.927254e-02  8.926702e-02  2.390833e-03   \n",
       "min    0.000000e+00  0.000000e+00  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "25%    0.000000e+00  2.173913e-01  5.977203e-03  0.000000e+00  0.000000e+00   \n",
       "50%    0.000000e+00  2.173913e-01  7.200445e-02  0.000000e+00  0.000000e+00   \n",
       "75%    0.000000e+00  6.666667e-01  1.431749e-01  0.000000e+00  0.000000e+00   \n",
       "max    1.000000e+00  1.000000e+00  1.000000e+00  1.000000e+00  1.000000e+00   \n",
       "\n",
       "       wrong_fragment        urgent           hot  num_failed_logins  \\\n",
       "count    4.898431e+06  4.898431e+06  4.898431e+06       4.898431e+06   \n",
       "mean     2.253783e-04  1.184053e-06  4.127078e-04       6.410216e-06   \n",
       "std      1.454819e-02  7.825853e-04  1.519020e-02       1.459882e-03   \n",
       "min      0.000000e+00  0.000000e+00  0.000000e+00       0.000000e+00   \n",
       "25%      0.000000e+00  0.000000e+00  0.000000e+00       0.000000e+00   \n",
       "50%      0.000000e+00  0.000000e+00  0.000000e+00       0.000000e+00   \n",
       "75%      0.000000e+00  0.000000e+00  0.000000e+00       0.000000e+00   \n",
       "max      1.000000e+00  1.000000e+00  1.000000e+00       1.000000e+00   \n",
       "\n",
       "          logged_in      ...            flag_S1       flag_S2       flag_S3  \\\n",
       "count  4.898431e+06      ...       4.898431e+06  4.898431e+06  4.898431e+06   \n",
       "mean   1.435290e-01      ...       1.086062e-04  3.286767e-05  1.020735e-05   \n",
       "std    3.506116e-01      ...       1.042087e-02  5.732939e-03  3.194878e-03   \n",
       "min    0.000000e+00      ...       0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "25%    0.000000e+00      ...       0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "50%    0.000000e+00      ...       0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "75%    0.000000e+00      ...       0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "max    1.000000e+00      ...       1.000000e+00  1.000000e+00  1.000000e+00   \n",
       "\n",
       "            flag_SF       flag_SH     label_DoS  label_Normal   label_Probe  \\\n",
       "count  4.898431e+06  4.898431e+06  4.898431e+06  4.898431e+06  4.898431e+06   \n",
       "mean   7.643933e-01  2.123129e-04  7.927783e-01  1.985964e-01  8.390850e-03   \n",
       "std    4.243774e-01  1.456941e-02  4.053158e-01  3.989435e-01  9.121648e-02   \n",
       "min    0.000000e+00  0.000000e+00  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "25%    1.000000e+00  0.000000e+00  1.000000e+00  0.000000e+00  0.000000e+00   \n",
       "50%    1.000000e+00  0.000000e+00  1.000000e+00  0.000000e+00  0.000000e+00   \n",
       "75%    1.000000e+00  0.000000e+00  1.000000e+00  0.000000e+00  0.000000e+00   \n",
       "max    1.000000e+00  1.000000e+00  1.000000e+00  1.000000e+00  1.000000e+00   \n",
       "\n",
       "          label_R2L     label_U2R  \n",
       "count  4.898431e+06  4.898431e+06  \n",
       "mean   2.298695e-04  4.491234e-06  \n",
       "std    1.515971e-02  2.119249e-03  \n",
       "min    0.000000e+00  0.000000e+00  \n",
       "25%    0.000000e+00  0.000000e+00  \n",
       "50%    0.000000e+00  0.000000e+00  \n",
       "75%    0.000000e+00  0.000000e+00  \n",
       "max    1.000000e+00  1.000000e+00  \n",
       "\n",
       "[8 rows x 58 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_normalized.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Shuffle data\n",
    "df_normalized = df_normalized.sample(n=len(df_normalized), random_state = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_normalized.to_csv('KDD.preProcessed.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
